# -*- coding: utf-8 -*-
"""
Created on Sat Dec 19 19:17:42 2020

@author: LSTM
"""
import os
from datetime import date
from datetime import timedelta
from pandas.tseries.offsets import MonthEnd
from pandas import to_datetime, read_pickle
import requests, json
import pandas as pd
from copy import deepcopy

class TjdDate():
    """
    实例化日期类,用于生成所需要定位的日期,默认是今日
    """

    def __init__(self, td=date.today()):
        super(TjdDate, self).__init__()
        # 本日,本月第一天,本月最后一天,上年本月第一天,上年本月最后一天
        self.today = to_datetime(td)
        self.month_first_day = date(self.today.year, self.today.month, 1)
        self.month_last_day = self.today + MonthEnd(0)
        self.last_month_last_day = self.month_first_day + timedelta(days=-1)
        self.last_month_first_day = date(self.last_month_last_day.year, self.last_month_last_day.month, 1)
        self.next_month_first_day = self.get_next_month_first_day()
        self.next_month_last_day = self.next_month_first_day + MonthEnd(0)
        self.last_year_month_first_day = date(self.today.year - 1, self.today.month, 1)
        self.last_year_month_last_day = self.last_year_month_first_day + MonthEnd(0)
        self.dt = {'td': self.today.strftime("%Y-%m-%d"),
                   'mfd': self.month_first_day.strftime("%Y-%m-%d"),
                   'mld': self.month_last_day.strftime("%Y-%m-%d"),
                   'lmfd': self.last_month_first_day.strftime("%Y-%m-%d"),
                   'lmld': self.last_month_last_day.strftime("%Y-%m-%d"),
                   'nmfd': self.next_month_first_day.strftime("%Y-%m-%d"),
                   'nmld': self.next_month_last_day.strftime("%Y-%m-%d"),
                   'lymfd': self.last_year_month_first_day.strftime("%Y-%m-%d"),
                   'lymld': self.last_year_month_last_day.strftime("%Y-%m-%d")}
        weekmask_egypt = "Mon Tue Wed Thu Fri"
        # holidays =  ["2011-01-01", datetime.datetime(2013, 5, 1), np.datetime64("2014-05-01"),]
        """
        holidays包含元旦、春节、元宵节、清明节、劳动节、端午节、中秋节、国庆节
        """
        holidays = ["2011-01-01", "2011-01-02", "2011-01-03", "2011-02-02", "2011-02-03", "2011-02-04","2011-02-05", "2011-02-06", "2011-02-07", "2011-02-08", 
                    "2011-04-03", "2011-04-04", "2011-04-05", "2011-04-30","2011-05-01", "2011-05-02", "2011-06-04", "2011-06-05", "2011-06-06", "2011-09-10", "2011-09-11", "2011-09-12",
                    "2011-10-01", "2011-10-02", "2011-10-03", "2011-10-04", "2011-10-05", "2011-10-06", "2011-10-07",
                    "2012-01-01", "2012-01-02", "2012-01-03", "2012-01-22", "2012-01-23", "2012-01-24","2012-01-25", "2012-01-26", "2012-01-27", "2012-01-28",
                    "2012-04-02", "2012-04-03", "2012-04-04", "2012-04-29", "2012-04-30","2012-05-01", "2012-06-22","2012-06-23", "2012-06-24",
                    "2012-09-30", "2012-10-01", "2012-10-02", "2012-10-03","2012-10-04", "2012-10-05", "2012-10-05", "2012-10-06", "2012-10-07",
                    "2013-01-01", "2013-01-02", "2013-01-03", "2013-02-09", "2013-02-10", "2013-02-11","2013-02-12", "2013-02-13", "2013-02-14", "2013-02-15",
                    "2013-04-04", "2013-04-05", "2013-04-06", "2013-04-29", "2013-04-30","2013-05-01", "2013-06-10","2013-06-11", "2013-06-12", "2013-09-19", "2013-09-20", "2013-09-21",
                    "2013-10-01","2013-10-02","2013-10-03", "2013-10-04", "2013-10-05", "2013-10-06", "2013-10-07",
                    "2014-01-01", "2014-01-02", "2014-01-03", "2014-01-04", "2014-01-05", "2014-01-29", "2014-01-30", "2014-01-31", "2014-02-01", "2014-02-02", "2014-02-03", "2014-02-04","2014-02-05", "2014-02-06",
                    "2014-02-14", "2014-02-15","2014-04-04",  "2014-04-05", "2014-04-06","2014-04-07","2014-04-29", "2014-04-30","2014-05-01", "2014-05-02", "2014-05-03", "2014-05-31", "2014-06-01","2014-06-02", "2014-09-08",
                    "2014-10-01","2014-10-02", "2014-10-03", "2014-10-04", "2014-10-05", "2014-10-06", "2014-10-07", "2014-10-08", "2014-10-09",
                    "2015-01-01", "2015-01-02", "2015-01-03","2015-02-18", "2015-02-19", "2015-02-20", "2015-02-21", "2015-02-22","2015-02-23", "2015-02-24",
                    "2015-04-05","2015-04-06", "2015-05-01", "2015-06-20", "2015-06-22", "2015-09-03","2015-09-04", "2015-09-27", 
                    "2015-10-01","2015-10-02", "2015-10-03", "2015-10-04", "2015-10-05", "2015-10-06", "2015-10-07",
                    "2016-01-01", "2016-02-07", "2016-02-08","2016-02-09", "2016-02-10", "2016-02-11", "2016-02-12", "2016-02-13",
                    "2016-04-04", "2016-05-01","2016-05-02", "2016-06-09","2016-06-10","2016-09-15", "2016-09-16", "2016-09-17",
                    "2016-10-01","2016-10-02", "2016-10-03", "2016-10-04", "2016-10-05", "2016-10-06", "2016-10-07",
                    "2017-01-01",  "2017-01-02","2017-01-27", "2017-01-28","2017-01-29", "2017-01-30", "2017-01-31", "2017-02-01", "2017-02-02",
                    "2017-04-02", "2017-04-03", "2017-04-04", "2017-05-01", "2017-05-28", "2017-05-29", "2017-05-30","2017-07-02",
                    "2017-10-01","2017-10-02", "2017-10-03", "2017-10-04", "2017-10-05", "2017-10-06", "2017-10-07", "2017-10-08",
                    "2018-01-01", "2018-02-15", "2018-02-16","2018-02-17", "2018-02-18", "2018-02-19", "2018-02-20", "2018-02-21",
                    "2018-04-05", "2018-04-06", "2018-04-07", "2018-04-29", "2018-04-30", "2018-05-01", "2018-06-18","2018-09-24",
                    "2018-10-01","2018-10-02", "2018-10-03", "2018-10-04", "2018-10-05", "2018-10-06", "2018-10-07","2018-12-31",
                    "2019-01-01", "2019-02-04", "2019-02-05","2019-02-06", "2019-02-07", "2019-02-08", "2019-02-09", "2019-02-10",
                    "2019-04-05", "2019-05-01",  "2019-05-02", "2019-05-03", "2019-06-07", "2019-09-13", 
                    "2019-10-01", "2019-10-02", "2019-10-03", "2019-10-04", "2019-10-05", "2019-10-06", "2019-10-07",
                    "2020-01-01", "2020-01-24", "2020-01-25","2020-01-26", "2020-01-27",  "2020-01-28", "2020-01-29", "2020-01-30","2020-01-31",
                    "2020-04-04", "2020-04-05", "2020-04-06", "2020-05-01", "2020-05-02", "2020-05-03", "2020-05-04", "2020-05-05", "2020-06-25", "2020-06-26", "2020-06-27", 
                    "2020-10-01", "2020-10-02", "2020-10-03", "2020-10-04", "2020-10-05", "2020-10-06", "2020-10-07","2020-10-08",
                    "2021-01-01", "2021-01-02", "2021-01-03","2021-02-11", "2021-02-12",  "2021-02-13", "2021-02-14", "2021-02-15", "2021-02-16", "2021-02-17",
                    "2021-04-03", "2021-04-04", "2021-04-05", "2021-05-01", "2021-05-02", "2021-05-03", "2021-05-04", "2021-05-05", "2021-06-12", "2021-06-13", "2021-06-14", "2021-09-19", "2021-09-20", "2021-09-21",
                    "2021-10-01", "2021-10-02", "2021-10-03", "2021-10-04", "2021-10-05", "2021-10-06", "2021-10-07"]
        bday_egypt = pd.offsets.CustomBusinessDay(holidays=holidays, weekmask=weekmask_egypt,)
        self.stock_freq_day = bday_egypt               

    def get_next_month_first_day(self):
        if self.today.month == 12:
            month = 1
            year = self.today.year + 1
        else:
            month = self.today.month + 1
            year = self.today.year
        return date(year, month, 1)


def current_time(date_format: str = "ms"):
    import datetime
    """
    获取当前时间字符串格式
    可选精度：年 year, 月 month, 日 day，小时 hour，分钟 minute，秒 second, 毫秒 ms
    """
    if date_format == "year":
        time_len = 4
    elif date_format == "month":
        time_len = 7
    elif date_format == "day":
        time_len = 10
    elif date_format == "hour":
        time_len = 13
    elif date_format == "minute":
        time_len = 16
    elif date_format == "second":
        time_len = 19
    elif date_format == "ms":
        time_len = 22
    elif date_format == "us":
        time_len = 26
    else:
        print("您输入的时间格式有误，请检查！")
        raise
    t = datetime.datetime.now()
    t_str = str(t)[:time_len]
    t_str = t_str.replace(':', '_')
    t_str = t_str.replace(' ', '_')
    t_str = t_str.replace('-', '_')
    t_str = t_str.replace('.', '_')
    return t_str


def get_file_list(path, file_list):
    if os.path.isfile(path):
        file_list.append(path)
    elif os.path.isdir(path):
        for s in os.listdir(path):
            new_path = os.path.join(path, s)
            get_file_list(new_path, file_list)
    return file_list


def drop_drop(data):
    print('drop before', data.shape)
    cols = data.columns.to_list()
    new_cols, exists_cols = [], []
    for i, c in enumerate(cols):
        if c not in exists_cols:
            new_cols.append(i)
            exists_cols.append(c)
    print('drop after', len(new_cols))
    return data.iloc[:, new_cols]

def real_path(pth):
    import os
    current_dir = os.getcwd()
    main_paths = []
    main_path_input = os.path.join(current_dir.split('pipeline')[0], 'input')
    main_paths.append(main_path_input)
    main_path_output = os.path.join(current_dir.split('pipeline')[0], 'output')
    for ss in ['args_result', 'feature_selection', 'model', 'selector', 'trainer',
               'collector', 'plot',
               'model_result', 'summary_result']:
        # print(ss)
        main_paths.append(os.path.join(main_path_output, ss))
    for main_path in main_paths:
        all_docs = get_file_list(main_path, [])
        for doc_str in all_docs:
            # print(doc_str)
            if pth in doc_str:
                print('lock>>>', doc_str)
                return doc_str


def check_path(pth):
    import os
    dir_path = os.path.split(pth)[0]
    if not os.path.exists(dir_path):
        # print('change dir >>> from', pth)
        if '../../' in pth:
            path = pth.replace('../../', '../')
        else:
            path = pth.replace('../', '../../')
        print('To:', path)
    else:
        path = pth
        # print(path)
    return path


def t_print(args):
    p = False
    if p:
        print(**args)


def output_dir():
    pth = os.getcwd()
    main_dir = pth.split('pipeline')[0]
    exists(main_dir)
    input_ = os.path.join(main_dir, 'input')
    exists(input_)
    output = os.path.join(main_dir, 'output')
    exists(output)
    for s in ['collector', 'constructor', 'plot', 'rolling_acc', 'space', 'path',
              'selector', 'to_tjd', 'trainer','collector_test','monitor','save_space','temp_file']:
        exists(os.path.join(output, s))


def exists(path):
    if not os.path.exists(path):
        print('已创建>>>', path)
        os.makedirs(path)


def output_exists():
    path = os.getcwd()
    main_dir = path.split('pipeline')[0]
    print('工作目录check', main_dir)
    
    return main_dir


def load_pickle(path):
    import pickle
    with open(path, 'rb') as fo:
        para = pickle.load(fo, encoding='bytes')
    return para


def save_pickle(paras, s_dir, long_str=False):
    import pickle
    if long_str:
        path = s_dir
    else:
        path = '../output/' + s_dir + '/' + s_dir + current_time() + '.pkl'
    path = check_path(path)
    print(path)
    with open(path, 'wb') as fo:
        pickle.dump(paras, fo)
    return path


def save_subspace(graph, space, subspace, subgraph):
    all_keys = []
    for keys_li in subgraph:
        for key in keys_li:
            if key not in all_keys:
                all_keys.append(key)
    d = {
        'graph': dict(graph),
        'space': space,
        'subgraph': subgraph,
        'subspace': subspace,
        'all_keys': all_keys,
        'time_s': current_time(),
    }
    s_dir = 'selector_space' if 'y_name' in dict(graph) else 'trainer_space'
    path = save_pickle(d, s_dir)
    return path

def send_msg(text_to_send, name_to_at):
    wx_url = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=765f9e09-edbe-4e54-82c3-8ad081519564"
    """艾特全部，并发送指定信息"""
    ater = "@" + name_to_at
    data = json.dumps({"msgtype": "text", "text": {"content": text_to_send, "mentioned_list":[ater]}})
    print(wx_url)
    r = requests.post(wx_url, data, auth=('Content-Type', 'application/json'))
    print(r.json)


def y_name_transfer(y_name, target_y_type):
    basic_y_name = "_".join(y_name.split("_")[:-1])
    if target_y_type == 'real':
        return basic_y_name + '_当月值'
    if target_y_type == 'cum_real':
        return basic_y_name + '_累计值'
    if target_y_type == 'yoy':
        return basic_y_name + '_当月同比'
    if target_y_type == 'cum_yoy':
        return basic_y_name + '_累计同比'
    if target_y_type == 'mom':
        return basic_y_name + '_当月环比'
    else:
        raise ValueError("输入有误，请检查")


class CheckEdges():
    def __init__(self, edges:dict, y_name:str, if_check_learner:bool = True, if_check_time_index:bool = True, if_check_dataset_len:bool = True, if_check_estimate_mode:bool = True, if_check_prediction_mode: bool = True):
        self.edges = edges
        self.if_check_learner = if_check_learner
        self.if_check_time_index = if_check_time_index
        self.if_check_dataset_len = if_check_dataset_len
        self.if_check_estimate_mode = if_check_estimate_mode
        self.if_check_prediction_mode = if_check_prediction_mode
        self.x = edges['x']
        self.y = edges['y']
        self.aim = edges['aim']
        self.source_y_type = edges["source_y_type"]
        self.target_y_type = edges["target_y_type"]
        self.estimate_mode = edges["estimate_mode"]
        self.estimate_value = edges["estimate_value"]
        self.learner = edges['trainer']['learner']
        if  "TjdDate" in  edges['Slicer1']['freq'][0]:
            self.freq = eval(edges['Slicer1']['freq'][0])
        else:
            self.freq = edges['Slicer1']['freq'][0]
        self.end_datetime = edges['end_datetime']
        self.periods = edges['Slicer1']['periods']
        self.test_len = edges['Slicer1']['test_len'][0]
        self.n_features = edges['Slicer2']['n_features']
        self.x_time_step = edges['Slicer2']['x_time_step']
        self.prediction = edges['Slicer2']['prediction'][0]
        self.y_time_step = edges['Slicer2']['y_time_step']
        self.valid_pct = edges['Slicer2']['valid_pct']
        self.deep_learning_model = edges['deep_learning']['model']
        self.x_value = self.x.loc[self.x[self.x.columns[0]].notnull(),:]
        self.y_time_index = self.y.index
        self.y_value_index = self.y.loc[self.y[y_name].notnull(),y_name].index

        self.x_min = pd.date_range(end = min(self.end_datetime), periods = max(self.periods) + max(self.y_time_step) + max(self.x_time_step) - 1, freq= self.freq).strftime('%F')[0]
        self.x_max = pd.date_range(start = max(self.end_datetime), periods = self.test_len, freq= self.freq).strftime('%F')[-1]
        self.min_train_len = round(min(self.periods)*(1- max(self.valid_pct)))
        self.min_valid_len = min(self.periods) - round(min(self.periods)*(1- min(self.valid_pct)))

        if self.prediction:
            self.y_max = pd.date_range(start = max(self.end_datetime), periods = max(self.y_time_step) + 1, freq = self.freq).strftime('%F')[-1]
            self.y_min = pd.date_range(end = min(self.end_datetime), periods = max(self.y_time_step) + max(self.periods) - 1,freq = self.freq).strftime('%F')[0]
            self.valid_y_max = max(self.end_datetime)
        else:
            self.y_max = max(self.end_datetime)
            self.y_min = pd.date_range(end = min(self.end_datetime), periods = max(self.periods) + 1, freq = self.freq).strftime('%F')[0]
            self.valid_y_max = pd.date_range(end = max(self.end_datetime), periods = 2, freq = self.freq).strftime('%F')[0]

        self.y_max_min_index = pd.date_range(start = self.y_min, end = self.valid_y_max, freq = self.freq).strftime('%F')

    def check_time_index(self):
        if self.x_min in self.x.index.strftime('%F') and self.x_max in self.x.index.strftime('%F'):
            print('The index of x is ok!')
        else:
            print(self.x_min)
            print(self.x_max)
            raise ValueError('The index of x is error!')
        if self.y_min in self.y_time_index.strftime('%F') and self.y_max in self.y_time_index.strftime('%F'):
            for item in self.y_max_min_index:
                if item in self.y_value_index:
                    pass
                else:
                    print(str(item) + " is not in y")
                    raise ValueError('The index of y is error!')
            if self.valid_y_max in self.y_value_index.strftime('%F'):
                print('The index of y is ok!')
            else:
                raise ValueError('valid_y_max %s is error!'%(self.valid_y_max))
        else:
            raise ValueError('The index of y is error!')
        

    def check_dataset_len(self):
        if self.min_train_len > 1 and self.min_valid_len > 1:
            print('The length of dataset is ok.')
        else:
            if self.min_train_len <= 1:
                raise ValueError('The length of trian is too small.')
            if self.min_valid_len <= 1:
                raise ValueError('The length of valid is too small.')
    
    def check_learner(self):
        if 'machine_learning' in self.learner:
            if len(self.x_time_step) == 1 and self.x_time_step[0] == 1:
                pass
            else:
                raise ValueError('learner or x_time_step is error!')
        else:
            pass
        print('learner is ok!') 
    
    def check_estimate_mode(self):
        if self.estimate_mode and self.aim == "test":
            raise ValueError('estimate_mode is true. aim must be download_model. Please check!')
        else:
            pass
        for key, value in self.estimate_value.items():
            for key_, value_ in value.items():
                if key_ in ["mae","mse","mae_y"]:
                    if value_[0] >= value_[1]:
                        raise ValueError("estimate_value {}:{}:{}".format(key, key_, value_) + " is error!")
                    else:
                        pass
                else:
                    if value_[0] <= value_[1]:
                        raise ValueError("estimate_value {}:{}:{}".format(key, key_, value_) + " is error!")
                    else:
                        pass
        print('estimate_mode is ok!')
    
    def check_prediction_mode(self):
        if self.prediction:
            pass
        else:
            if self.test_len * max(self.y_time_step) == 1:
                pass
            else:
                raise ValueError("Now is prediction mode! test_len and y_time_step must be 1 !")
        print('prediction mode is ok!')

    def process(self):
        if self.if_check_learner:
            self.check_learner()
        if self.x_value.isnull().sum().sum() > 0:
            raise ValueError("x has null.")
        if self.check_time_index:
            self.check_time_index()
        if self.if_check_dataset_len:
            self.check_dataset_len()
        if self.if_check_estimate_mode:
            self.check_estimate_mode()
        if self.if_check_prediction_mode:
            self.check_prediction_mode()
        print('Edges is ok!')
        
        

def get_all_collector(ready=False):
    dir_path = "..\..\output\collector"
    dir_path = check_path(dir_path)
    all_paths = os.listdir(dir_path)
    c_all_path = os.path.join(dir_path, 'all_collector.pkl')
    for path in all_paths:
        if 'all_collector' in path:
            print('yes')
            path_data_dict = read_pickle(c_all_path)
            ready = True
    if not ready:
        path_data_dict = {}
        for path in all_paths:
            if '.pkl' in path:
                s_path = os.path.join(dir_path, path)
                data = read_pickle(s_path)
                y_name = data['y_name']
                print(y_name, '>', s_path)
                path_data_dict[y_name] = s_path
        save_pickle(path_data_dict, c_all_path, long_str=True)
    return path_data_dict



if __name__ == "__main__":
    from container.edges import edges
    edges_check = CheckEdges(edges = edges, y_name = '贷款加权利率_当月同比').process()
    #edges_check = CheckEdges(edges = edges, y_name = 'M2', if_check_time_index = False, if_check_dataset_len = False) 

    # d = TjdDate('2021-04-06')
    # send_msg('自定义文字测试', 'all')
    # pth1 = real_path('y集合')
    # args_path = '../output/args_result/args_com_hcom_gcom.pkl'
    # c = check_path(args_path)
# =============================================================================
#     d = TjdDate()
#     pth1 = real_path('y集合')
#     args_path = '../output/args_result/args_com_hcom_gcom.pkl'
#     c = check_path(args_path)
# =============================================================================
